{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "okHucoMFSm_4"
      },
      "outputs": [],
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CDftOC_sS4a5"
      },
      "outputs": [],
      "source": [
        "#@title Setup\n",
        "!pip install git+https://github.com/google-research/google-research.git#subdirectory=tide_nlp\n",
        "!python -m spacy download en_core_web_sm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tgKvMt7cioJA"
      },
      "outputs": [],
      "source": [
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "executionInfo": {
          "elapsed": 638,
          "status": "ok",
          "timestamp": 1694798918744,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "fayZ8IUse4H6"
      },
      "outputs": [],
      "source": [
        "#@title Imports\n",
        "import sys\n",
        "\n",
        "import bs4\n",
        "import pandas as pd\n",
        "import requests\n",
        "import spacy\n",
        "\n",
        "import tide_nlp as tide_nlp\n",
        "from tide_nlp import identity_annotator as ia\n",
        "from tide_nlp.entity_annotator import identity_spacy_annotator as i_spacy_a\n",
        "from tide_nlp.entity_annotator import non_ptc_annotator as non_ptc_a\n",
        "from tide_nlp.entity_annotator import ptc_annotator as ptc_a\n",
        "from tide_nlp.entity_annotator import ptc_helper as ptc\n",
        "from tide_nlp.entity_annotator import spacy_annotator as spacy_a\n",
        "from tide_nlp.lexicon import tidal_lexicon as lex\n",
        "from tide_nlp import tidal_util\n",
        "from tide_nlp.tokenizer import spacy_tokenizer as tok"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "executionInfo": {
          "elapsed": 4866,
          "status": "ok",
          "timestamp": 1694798925733,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "3eJNpJKD8E4h"
      },
      "outputs": [],
      "source": [
        "#@title Initialize TIDAL lexicon\n",
        "\n",
        "tidal_lexicon_df = tidal_util.read_tidal()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "executionInfo": {
          "elapsed": 291,
          "status": "ok",
          "timestamp": 1694798927425,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "wV-Zu3HYWrNi"
      },
      "outputs": [],
      "source": [
        "#@title Download person noun lexicon\n",
        "PERSON_NOUN_LEXICON_URLS = ['https://en.wiktionary.org/w/index.php?title=Category:English_terms_of_address',\n",
        "                            'https://en.wiktionary.org/w/index.php?title=Category:English_terms_of_address\u0026pagefrom=SNOOKUMS%0Asnookums#mw-pages']\n",
        "\n",
        "person_noun_terms = []\n",
        "\n",
        "for url in PERSON_NOUN_LEXICON_URLS:\n",
        "  response = requests.get(url)\n",
        "  soup = bs4.BeautifulSoup(response.content, 'html.parser')\n",
        "  mw_category_divs = soup.find_all('div', {'class': 'mw-category-group'})\n",
        "\n",
        "  for div in mw_category_divs:\n",
        "    for a in div.find_all('a'):\n",
        "      noun = a.text.lower()\n",
        "\n",
        "      # Remove terms that are less than 3 characters (eg Mt)\n",
        "      if len(noun) \u003c 3:\n",
        "        continue\n",
        "      # Remove terms that have a period (eg Mr. President)\n",
        "      if '.' in noun:\n",
        "        continue\n",
        "\n",
        "      person_noun_terms.append(noun)\n",
        "\n",
        "person_lexicon_df = pd.DataFrame(person_noun_terms, columns=['noun'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "executionInfo": {
          "elapsed": 1396,
          "status": "ok",
          "timestamp": 1694798932132,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "C1GYAs8676Ll"
      },
      "outputs": [],
      "source": [
        "#@title Configure annotation options\n",
        "\n",
        "model_path = 'en_core_web_sm'\n",
        "nlp = spacy.load(model_path)\n",
        "\n",
        "lexicon = lex.TidalLexicon(tidal_lexicon_df)\n",
        "tokenizer = tok.SpacyTokenizer(nlp)\n",
        "\n",
        "person_helper_lexicon = ptc.PersonMentionHelper(nlp, person_lexicon_df)\n",
        "ptc_lexicon_annotator = ptc_a.PtcAnnotator(person_helper_lexicon)\n",
        "non_ptc_lexicon_annotator = non_ptc_a.NonPtcAnnotator(person_helper_lexicon)\n",
        "\n",
        "person_helper_similarity = ptc.PersonMentionHelper(nlp, use_nltk_similarity=True)\n",
        "ptc_similarity_annotator = ptc_a.PtcAnnotator(person_helper_similarity)\n",
        "non_ptc_similarity_annotator = non_ptc_a.NonPtcAnnotator(person_helper_similarity)\n",
        "\n",
        "spacy_annotator = spacy_a.SpacyAnnotator(nlp)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "executionInfo": {
          "elapsed": 3310,
          "status": "ok",
          "timestamp": 1694798938364,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "gOrqhNVddwdz",
        "outputId": "d6cba2c1-0a08-4333-d0be-f41fd6db9de2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "identity groups:  ['Race_Nationality_Ethnicity']\n",
            "identity terms:  ['black' 'americans']\n",
            "identity group-term dictionary:\n",
            " {'Race_Nationality_Ethnicity': ['black', 'americans']}\n",
            "annotation candidates:\n",
            " ,mention.tokens.limit,mention.tokens.start,mention.type,ptc.identity_term,ptc.identity_token,ptc.person_term,ptc.person_token,ptc.ptc_term,ptc.text,IsPTCTerm,bytes.start,IdentityGroup,IdentitySubgroup,HasNonIdentityMeaning,token.lemma,token.tag,IsRootTerm,IdentityGroup_Connotation_ConvergenceScore,token.dependencyHead.index,text,token.index,token.dependencyLabel,bytes.limit,token.pos,Term,Connotation,PossibleNonIdentity\n",
            "0,13.0,12.0,IDENTITY_LEXICON,black,11.0,americans,12.0,black americans,black,False,55,Race_Nationality_Ethnicity,Black,True,black,JJ,True,1.0,12,black,11,amod,60,ADJ,black,\"('NEUTRAL',)\",False\n",
            "1,,,,,,,,,,False,10,Race_Nationality_Ethnicity,White,True,white,JJ,True,1.0,3,white,2,amod,15,ADJ,white,\"('NEUTRAL',)\",\n",
            "2,,,,,,,,,,False,55,Race_Nationality_Ethnicity,Black,True,black,JJ,True,1.0,12,black,11,amod,60,ADJ,black,\"('NEUTRAL',)\",\n",
            "3,,,,,,,,,,False,61,Race_Nationality_Ethnicity,American,True,american,NNS,False,1.0,10,americans,12,pobj,70,NOUN,americans,\"('NEUTRAL',)\",\n",
            "0,,,,,,,,,,False,10,Race_Nationality_Ethnicity,White,True,white,JJ,True,1.0,3,white,2,amod,15,ADJ,white,\"('NEUTRAL',)\",True\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  unambiguous_non_identity_df['PossibleNonIdentity'] = True\n"
          ]
        }
      ],
      "source": [
        "#@title Test annotation\n",
        "\n",
        "# This uses a simple token-based annotation logic to determine whether an\n",
        "# identity term is modifying a known person noun based on the lexicon.\n",
        "entity_annotators = [ptc_lexicon_annotator, spacy_annotator]\n",
        "non_entity_annotators = [non_ptc_lexicon_annotator]\n",
        "\n",
        "annotator = ia.IdentityAnnotator(lexicon=lexicon,\n",
        "                                 tokenizer=tokenizer,\n",
        "                                 entity_annotators=entity_annotators,\n",
        "                                 non_entity_annotators=non_entity_annotators)\n",
        "\n",
        "test_text = '''Love your white car! Transaxle FWD cards are great for Black Americans.'''\n",
        "\n",
        "groups, terms, group_term_dict, df = annotator.annotate(test_text.lower())\n",
        "\n",
        "print('identity groups: ', groups)\n",
        "print('identity terms: ', terms)\n",
        "print('identity group-term dictionary:\\n', group_term_dict)\n",
        "print('annotation candidates:\\n', df.to_csv())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "executionInfo": {
          "elapsed": 16,
          "status": "ok",
          "timestamp": 1694798945204,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "DKaHlMLPCpBW"
      },
      "outputs": [],
      "source": [
        "#@title Utility functions for bulk annotation\n",
        "\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "\n",
        "def annotate_example_row_lib(lib, row, text_column='comment_text'):\n",
        "  text = row[text_column].lower()\n",
        "  groups, terms, group_term_dict, df = lib.annotate(text)\n",
        "\n",
        "  if len(groups):\n",
        "    row['identity_groups'] = groups\n",
        "    row['identity_terms'] = terms\n",
        "    row['annotation_group_term_dict'] = group_term_dict\n",
        "\n",
        "  if not df.empty:\n",
        "    row['df'] = df.to_dict('records')\n",
        "\n",
        "  return row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "executionInfo": {
          "elapsed": 7046,
          "status": "ok",
          "timestamp": 1694798960481,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "dBc7CWk1YZjg",
        "outputId": "ba0b2f5c-55be-4184-dd8e-7e891838e1f4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  219M  100  219M    0     0  40.8M      0  0:00:05  0:00:05 --:--:-- 44.7M\n"
          ]
        }
      ],
      "source": [
        "#@title Fetch CivilComments data\n",
        "\n",
        "!curl https://storage.googleapis.com/civil_comments_dataset/validate_df_processed.csv -o /tmp/civil_comments.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "executionInfo": {
          "elapsed": 36749,
          "status": "ok",
          "timestamp": 1694799002385,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "uz-yHgw2espf",
        "outputId": "f8a1f8c4-bcea-47ed-ec13-356a04e82845"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 20/20 [00:30\u003c00:00,  1.51s/it]\n"
          ]
        }
      ],
      "source": [
        "#@title Annotate a sample of CivilComments\n",
        "cc_df = pd.read_csv('/tmp/civil_comments.csv', usecols=['comment_text']).sample(20, random_state=789)\n",
        "\n",
        "annotated_cc_df = cc_df.progress_apply(lambda x: annotate_example_row_lib(annotator, x), axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "height": 3592
        },
        "executionInfo": {
          "elapsed": 713,
          "status": "ok",
          "timestamp": 1694799003173,
          "user": {
            "displayName": "",
            "userId": ""
          },
          "user_tz": 420
        },
        "id": "p9NaK6erespg",
        "outputId": "e82d5710-e2b1-4ca5-9f4c-0b89f35fae95"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  \u003cdiv id=\"df-891a335c-1da8-4052-9d8f-d56ff97e6039\" class=\"colab-df-container\"\u003e\n",
              "    \u003cdiv\u003e\n",
              "\u003cstyle scoped\u003e\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "\u003c/style\u003e\n",
              "\u003ctable border=\"1\" class=\"dataframe\"\u003e\n",
              "  \u003cthead\u003e\n",
              "    \u003ctr style=\"text-align: right;\"\u003e\n",
              "      \u003cth\u003e\u003c/th\u003e\n",
              "      \u003cth\u003eannotation_group_term_dict\u003c/th\u003e\n",
              "      \u003cth\u003ecomment_text\u003c/th\u003e\n",
              "      \u003cth\u003edf\u003c/th\u003e\n",
              "      \u003cth\u003eidentity_groups\u003c/th\u003e\n",
              "      \u003cth\u003eidentity_terms\u003c/th\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/thead\u003e\n",
              "  \u003ctbody\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e705165\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eCut the BS. Parental leave is paid for through...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e523627\u003c/th\u003e\n",
              "      \u003ctd\u003e{'SOGIESC': ['woman']}\u003c/td\u003e\n",
              "      \u003ctd\u003eTexting requires taking ones eyes off the road...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 211, 'Ide...\u003c/td\u003e\n",
              "      \u003ctd\u003e[SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[woman]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e698888\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eDavid Boyle is correct. To big a project with ...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e234859\u003c/th\u003e\n",
              "      \u003ctd\u003e{'SOGIESC': ['women', 'males', 'men', 'woman',...\u003c/td\u003e\n",
              "      \u003ctd\u003eSomeone said, \"Jesus picked males because in h...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 28, 'Iden...\u003c/td\u003e\n",
              "      \u003ctd\u003e[SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[males, women, woman, her, men]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e363171\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003e\"...Dave Nichol knew how to tell a story and h...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 213, 'Ide...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e698236\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eokay folks here is the law in BC\\n\\nup until 2...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e707493\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eLeave it to trump to cut out the middleman.\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e560366\u003c/th\u003e\n",
              "      \u003ctd\u003e{'SOGIESC': ['her']}\u003c/td\u003e\n",
              "      \u003ctd\u003eMy residency status doesn't mean that your ign...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 251, 'Ide...\u003c/td\u003e\n",
              "      \u003ctd\u003e[SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[her]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e297069\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eFunny, how Ohio and Virginia are marching to t...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e465256\u003c/th\u003e\n",
              "      \u003ctd\u003e{'SOGIESC': ['her', 'she']}\u003c/td\u003e\n",
              "      \u003ctd\u003eMake her drink 1/4 of it. If she will OK.\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 5, 'Ident...\u003c/td\u003e\n",
              "      \u003ctd\u003e[SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[her, she]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e629680\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eHe is not a person.  He is God!!!\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e293699\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eI based my comment on Dispatch stories that re...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e492065\u003c/th\u003e\n",
              "      \u003ctd\u003e{'Race_Nationality_Ethnicity': ['black', 'whit...\u003c/td\u003e\n",
              "      \u003ctd\u003eWho was the BLACK guy who fired the same kind ...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'mention.tokens.limit': 5.0, 'mention.tokens...\u003c/td\u003e\n",
              "      \u003ctd\u003e[Race_Nationality_Ethnicity, SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[black, whites, men, women, black guy]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e131239\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eThe average margin for a good restaurant is ar...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e429586\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eCavs should've picked him up last year instead...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e567684\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eIf so, why was their only action to try and ba...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e447632\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eI'd say we already have that fiscal nightmare.\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e669132\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eA very different accident too.   A plan loaded...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e632091\u003c/th\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eHowever, wealth should not be attained through...\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "      \u003ctd\u003eNaN\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "    \u003ctr\u003e\n",
              "      \u003cth\u003e554725\u003c/th\u003e\n",
              "      \u003ctd\u003e{'SOGIESC': ['her']}\u003c/td\u003e\n",
              "      \u003ctd\u003eNot making any excuses for GOP primary voters ...\u003c/td\u003e\n",
              "      \u003ctd\u003e[{'IsPTCTerm': False, 'bytes.start': 150, 'Ide...\u003c/td\u003e\n",
              "      \u003ctd\u003e[SOGIESC]\u003c/td\u003e\n",
              "      \u003ctd\u003e[her]\u003c/td\u003e\n",
              "    \u003c/tr\u003e\n",
              "  \u003c/tbody\u003e\n",
              "\u003c/table\u003e\n",
              "\u003c/div\u003e\n",
              "    \u003cdiv class=\"colab-df-buttons\"\u003e\n",
              "\n",
              "  \u003cdiv class=\"colab-df-container\"\u003e\n",
              "    \u003cbutton class=\"colab-df-convert\" onclick=\"convertToInteractive('df-891a335c-1da8-4052-9d8f-d56ff97e6039')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\"\u003e\n",
              "\n",
              "  \u003csvg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\"\u003e\n",
              "    \u003cpath d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/\u003e\n",
              "  \u003c/svg\u003e\n",
              "    \u003c/button\u003e\n",
              "\n",
              "  \u003cstyle\u003e\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  \u003c/style\u003e\n",
              "\n",
              "    \u003cscript\u003e\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-891a335c-1da8-4052-9d8f-d56ff97e6039 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-891a335c-1da8-4052-9d8f-d56ff97e6039');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '\u003ca target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb\u003edata table notebook\u003c/a\u003e'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    \u003c/script\u003e\n",
              "  \u003c/div\u003e\n",
              "\n",
              "\n",
              "\u003cdiv id=\"df-29f9acb4-d1ee-4115-a2aa-cfe4bb8bf855\"\u003e\n",
              "  \u003cbutton class=\"colab-df-quickchart\" onclick=\"quickchart('df-29f9acb4-d1ee-4115-a2aa-cfe4bb8bf855')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\"\u003e\n",
              "\n",
              "\u003csvg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\"\u003e\n",
              "    \u003cg\u003e\n",
              "        \u003cpath d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/\u003e\n",
              "    \u003c/g\u003e\n",
              "\u003c/svg\u003e\n",
              "  \u003c/button\u003e\n",
              "\n",
              "\u003cstyle\u003e\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "\u003c/style\u003e\n",
              "\n",
              "  \u003cscript\u003e\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() =\u003e {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-29f9acb4-d1ee-4115-a2aa-cfe4bb8bf855 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  \u003c/script\u003e\n",
              "\u003c/div\u003e\n",
              "    \u003c/div\u003e\n",
              "  \u003c/div\u003e\n"
            ],
            "text/plain": [
              "                               annotation_group_term_dict                                       comment_text                                                 df                        identity_groups                          identity_terms\n",
              "705165                                                NaN  Cut the BS. Parental leave is paid for through...                                                NaN                                    NaN                                     NaN\n",
              "523627                             {'SOGIESC': ['woman']}  Texting requires taking ones eyes off the road...  [{'IsPTCTerm': False, 'bytes.start': 211, 'Ide...                              [SOGIESC]                                 [woman]\n",
              "698888                                                NaN  David Boyle is correct. To big a project with ...                                                NaN                                    NaN                                     NaN\n",
              "234859  {'SOGIESC': ['women', 'males', 'men', 'woman',...  Someone said, \"Jesus picked males because in h...  [{'IsPTCTerm': False, 'bytes.start': 28, 'Iden...                              [SOGIESC]         [males, women, woman, her, men]\n",
              "363171                                                NaN  \"...Dave Nichol knew how to tell a story and h...  [{'IsPTCTerm': False, 'bytes.start': 213, 'Ide...                                    NaN                                     NaN\n",
              "698236                                                NaN  okay folks here is the law in BC\\n\\nup until 2...                                                NaN                                    NaN                                     NaN\n",
              "707493                                                NaN        Leave it to trump to cut out the middleman.                                                NaN                                    NaN                                     NaN\n",
              "560366                               {'SOGIESC': ['her']}  My residency status doesn't mean that your ign...  [{'IsPTCTerm': False, 'bytes.start': 251, 'Ide...                              [SOGIESC]                                   [her]\n",
              "297069                                                NaN  Funny, how Ohio and Virginia are marching to t...                                                NaN                                    NaN                                     NaN\n",
              "465256                        {'SOGIESC': ['her', 'she']}          Make her drink 1/4 of it. If she will OK.  [{'IsPTCTerm': False, 'bytes.start': 5, 'Ident...                              [SOGIESC]                              [her, she]\n",
              "629680                                                NaN                  He is not a person.  He is God!!!                                                NaN                                    NaN                                     NaN\n",
              "293699                                                NaN  I based my comment on Dispatch stories that re...                                                NaN                                    NaN                                     NaN\n",
              "492065  {'Race_Nationality_Ethnicity': ['black', 'whit...  Who was the BLACK guy who fired the same kind ...  [{'mention.tokens.limit': 5.0, 'mention.tokens...  [Race_Nationality_Ethnicity, SOGIESC]  [black, whites, men, women, black guy]\n",
              "131239                                                NaN  The average margin for a good restaurant is ar...                                                NaN                                    NaN                                     NaN\n",
              "429586                                                NaN  Cavs should've picked him up last year instead...                                                NaN                                    NaN                                     NaN\n",
              "567684                                                NaN  If so, why was their only action to try and ba...                                                NaN                                    NaN                                     NaN\n",
              "447632                                                NaN     I'd say we already have that fiscal nightmare.                                                NaN                                    NaN                                     NaN\n",
              "669132                                                NaN  A very different accident too.   A plan loaded...                                                NaN                                    NaN                                     NaN\n",
              "632091                                                NaN  However, wealth should not be attained through...                                                NaN                                    NaN                                     NaN\n",
              "554725                               {'SOGIESC': ['her']}  Not making any excuses for GOP primary voters ...  [{'IsPTCTerm': False, 'bytes.start': 150, 'Ide...                              [SOGIESC]                                   [her]"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "annotated_cc_df"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "last_runtime": {
        "build_target": "",
        "kind": "local"
      },
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
